//数据清洗，将html文件的标签去掉
#include <iostream>
#include <boost/filesystem.hpp>
#include <fstream>
#include <string>
#include <vector>
#include "util.hpp"
#include"log.hpp"

typedef struct DocInfo DocInfo_t;

const std::string src_path = "./data/input";            //我们处理的文件路径
const std::string raw_path = "./data/raw_html/raw.txt"; //我们输出的文件路径

bool enumFile(const std::string &src, std::vector<std::string> *file);
bool parseFile(const std::vector<std::string> &file, std::vector<DocInfo_t> *outPut);
bool saveFile(const std::vector<DocInfo_t> &outPut, const std::string& path);

//提取出的文件类型
typedef struct DocInfo
{
    std::string _title; //文件标题
    std::string _body;  //文件内容
    std::string _url;   //文件的url
} DocInfo_t;

bool getTitie(const std::string &cur, std::string *title)
{
    std::size_t begin = cur.find("<title>");
    //先判断有没有
    if (begin == std::string::npos)
    {
        LOG(ERROR,"get title error!");
        return false;
    }
    std::size_t end = cur.find("</title>");

    if (end == std::string::npos)
    {
        LOG(ERROR,"get title error!");
        return false;
    }

    begin += std::string("<title>").size();
    if (begin > end)
    {
        LOG(ERROR,"get title error!");
        return false;
    } //判断截取的是否有效

    *title = cur.substr(begin, end - begin);

    // for debug
    // std::cout << *title << std::endl;

    return true;
}

bool getBody(const std::string &cur, std::string *body)
{
    //基于状态机的操作
    enum status
    {
        LABLE,  //标签
        CONTENT //正文
    };

    enum status st = LABLE;

    for (auto &c : cur)
    {
        switch (st)
        {
        case LABLE:
            if (c == '>')
            {
                st = CONTENT;
            }
            break;
        case CONTENT:
            if (c == '<')
            {
                st = LABLE;
            }
            else
            {
                *body += c;
            }

            break;
        default:
            break;
        }
    }

    // for debug
    //std::cout << *body << std::endl;
    return true;
}

bool getUrl(const std::string &filePath, std::string *url)
{
    std::string head_url="https://www.boost.org/doc/libs/1_79_0/doc/html";
    std::string tail_url=filePath.substr(src_path.size());
    *url=head_url+tail_url;

    return true;
}

bool enumFile(const std::string &src, std::vector<std::string> *file)
{
    // boost中的文件处理
    namespace bf = boost::filesystem;

    bf::path path(src);
    //判断路径是否存在
    if (!bf::exists(path))
    {
        LOG(ERROR,"file not found");
        return false;
    }

    bf::recursive_directory_iterator end; //空迭代器表示文件结束
    for (bf::recursive_directory_iterator iter(path); iter != end; iter++)
    {
        if (!bf::is_regular_file(*iter)) //判断是不是普通文件
        {
            continue;
        }

        if (iter->path().extension() != ".html") //判断是不是html文件
        {
            continue;
        }

        file->push_back(iter->path().string());
        // for debug
        // std::cout<<iter->path().string()<<std::endl;
    }
    return true;
}

//for debug
// void showDoc(const DocInfo_t& doc)
// {
//     std::cout<<"title:"<<doc._title<<std::endl;
//     std::cout<<"body:"<<doc._body<<std::endl;
//     std::cout<<"url:"<<doc._url<<std::endl;
// }

bool parseFile(const std::vector<std::string> &file, std::vector<DocInfo_t> *outPut)
{
    // 2.1读取文件(用c++io流进行读取)
    //读取文件操作进行一下封装
    for (auto &path : file)
    {
        std::string tmpout;
        DocInfo_t tmpDoc;
        if (!ns_util::FileUtil::readFile(path, &tmpout))
        {
            LOG(ERROR,"file not found");
            return false;
        }

        // 2.2获得标签

        if (!getTitie(tmpout, &tmpDoc._title))
        {
            LOG(ERROR,"title erase error!");
            return false;
        }
        // 2.3获得正文

        if (!getBody(tmpout, &tmpDoc._body))
        {
            LOG(ERROR,"body erase error!");
            return false;
        }
        // 2.4获得文件对应官网中的url
        if (!getUrl(path, &tmpDoc._url))
        {
            LOG(ERROR,"url erase error!");
            return false;
        }

        outPut->push_back(std::move(tmpDoc));
        // for debug

        // showDoc(tmpDoc);
        // break;
    }

    return true;
}

bool saveFile(const std::vector<DocInfo_t> &outPut, const std::string& path)
{
#define SEP '\3'//分隔符区分每个文件的哪个部分，\n区分一个文件
    std::ofstream ofs(path,std::ios::out | std::ios::binary);

    if(!ofs.is_open())
    {
        LOG(ERROR,"open output error!");
        return false;
    }
    for(auto& doc:outPut)
    {
        std::string outstr;
        outstr+=doc._title;
        outstr+=SEP;
        outstr+=doc._body;
        outstr+=SEP;
        outstr+=doc._url;
        outstr+='\n';
        ofs.write(outstr.c_str(),outstr.size());
    }
    ofs.close();
    return true;
#undef SEP
}

int main()
{
    // 1.首先将所有的文件路径保存，方便后续的读取
    std::vector<std::string> filePath; //放置所有待文件路径

    if (!enumFile(src_path, &filePath))
    {
        LOG(ERROR,"enum error!");
        return 1;
    }

    // 2.文件的处理解析
    std::vector<DocInfo_t> outPut;

    if (!parseFile(filePath, &outPut))
    {
        LOG(ERROR,"parse error!");
        return 2;
    }

    // 3.处理后的文件保存

    if (!saveFile(outPut, raw_path))
    {
        LOG(ERROR,"save error!");
        return 3;
    }

    LOG(NORMAL,"parser success!");
    return 0;
}
